/*
 * Hyuga: A Density-Grid Stream Clustering Platform.
 * Copyright (C) 2014 PUC-Rio/Laboratory for Advanced Collaboration
 *
 * Hyuga is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Hyuga is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Hyuga.  If not, see <http://www.gnu.org/licenses/>.
 */
package br.pucrio.inf.lac.konoha.hyuga.others.csv;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;

import br.pucrio.inf.lac.konoha.hyuga.core.Output;
import br.pucrio.inf.lac.konoha.hyuga.core.output.CSVEntry;
import br.pucrio.inf.lac.konoha.hyuga.core.output.OutputType;

public class CSVDstreamRandIndex {
    File dg2cepFile;
    File dbscanFile;
    PrintWriter randIndexFW;
    
    Output outSingleton;
    
    private TreeMap<Long, List<CSVEntry>> dg2cepSecBySecEntries;
    private TreeMap<Long, List<CSVEntry>> dbscanSecBySecEntries;
    
    public static final double randLowThreshold  = 0.5;
    public static final double randCompThreshold = 0.5;
        
    public CSVDstreamRandIndex(String dg2cep, String dbscan, String output) {
        dg2cepFile = new File(dg2cep);
        dbscanFile = new File(dbscan);
        outSingleton = Output.getInstance();
        
        if (!dg2cepFile.exists()) {
            System.out.println("File " + dg2cepFile.getName() + " does not exists.");
            System.exit(-1);
        }

        if (!dbscanFile.exists()) {
            System.out.println("File " + dbscanFile.getName() + " does not exists.");
            System.exit(-1);
        }

        try {
            File randIndexFile = new File(output + "randIndex.csv");
            randIndexFile.createNewFile();
            randIndexFW = new PrintWriter(new BufferedWriter(new FileWriter(randIndexFile)));
        } catch (IOException ex) {
            System.out.println("Could not create output files");
            System.exit(-1);
        }
    }
    
    public CSVEntry getDSTREAMEntry(String csvLine) {
        // FirstDivision [0] timestamp,type,[cell],size
        // FirstDivision [1] cellcontent
        String[] firstDivision = csvLine.split("\"");
        String[] entrydata = firstDivision[0].split(",");
        long timestamp = Long.parseLong(entrydata[0]);
        OutputType type = OutputType.valueOf(entrydata[1]);
        
        if (type != OutputType.NONE) {
            int size = Integer.parseInt(entrydata[2]);

            if (size == 0) {
                return new CSVEntry(timestamp, OutputType.NONE);
            }
            Set<String> contentSet = new HashSet<String>();
            for (String node : firstDivision[1].split(",")) {
                contentSet.add(node);
            }
            
            return new CSVEntry(timestamp,type,size,csvLine,contentSet);
        } else {
            return new CSVEntry(timestamp, type);
        }
    }
    
    public CSVEntry getDBSCANEntry(String csvLine) {
        // FirstDivision [0] timestamp,snapshotsize,totaltime,size
        // FirstDivision [1] cellcontent
        String[] firstDivision = csvLine.split("\"");
        
        String[] entrydata = firstDivision[0].split(",");
        long timestamp = Long.parseLong(entrydata[0]);
        OutputType type = OutputType.DBSCAN;
        int size = Integer.parseInt(entrydata[3]);

        Set<String> contentSet = new HashSet<String>();
        for (String node : firstDivision[1].split(",")) {
            contentSet.add(node);
        }
                
        return new CSVEntry(timestamp,type,size,csvLine,contentSet);
    }

    private TreeMap<Long,List<CSVEntry>> dstreamSecBySecParser(BufferedReader dstreamBuffer) {
        TreeMap<Long,List<CSVEntry>> dstreamSecBySecEntries = new TreeMap<Long, List<CSVEntry>>();
        String dstreamCSVLine = "";
        
        try {
            while((dstreamCSVLine = dstreamBuffer.readLine()) != null) {
                CSVEntry dstreamEntry = getDSTREAMEntry(dstreamCSVLine);
                long entryTimestamp = dstreamEntry.getTimestamp();

                if (!dstreamSecBySecEntries.containsKey(entryTimestamp)) {
                    dstreamSecBySecEntries.put(entryTimestamp, new ArrayList<CSVEntry>());
                } 
                
                List<CSVEntry> clusterListInThisSecond = dstreamSecBySecEntries.get(entryTimestamp);
                
                if (dstreamEntry.getType() == OutputType.NONE) continue;
                else {
                    clusterListInThisSecond.add(dstreamEntry);
                }
            }
        } catch (IOException e) {
            System.out.println("Exception at reading dstream output file");
        }
        
        return dstreamSecBySecEntries;
    } 
    
   private TreeMap<Long,List<CSVEntry>> dbscanSecBySecParser(BufferedReader dbscanBuffer) {
       TreeMap<Long,List<CSVEntry>> dbscanSecBySecEntries = new TreeMap<Long, List<CSVEntry>>();
       String dbscanCSVLine = "";
       
       try {
           while((dbscanCSVLine = dbscanBuffer.readLine()) != null) {
               CSVEntry dbscanEntry = getDBSCANEntry(dbscanCSVLine);
               long entryTimestamp = dbscanEntry.getTimestamp();

               if (!dbscanSecBySecEntries.containsKey(entryTimestamp)) {
                   dbscanSecBySecEntries.put(entryTimestamp, new ArrayList<CSVEntry>());
               }
               
               List<CSVEntry> clusterListInThisSecond = dbscanSecBySecEntries.get(entryTimestamp);
               clusterListInThisSecond.add(dbscanEntry);
           }
       } catch (IOException e) {
           System.out.println("Exception at reading dbscan output file");
       }
       
       return dbscanSecBySecEntries;
   }
    
    private void updateClusters(CSVEntry dg2cepEntry, List<CSVEntry> currentClusters) {
        Iterator<CSVEntry> it = currentClusters.iterator();
        while (it.hasNext()) {
            if (randIndex(dg2cepEntry, it.next()) > randCompThreshold) {
                it.remove();
            }
        }
        currentClusters.add(dg2cepEntry);
     }
    
    private CSVEntry getHigherRandIndex(CSVEntry dg2cepEntry, List<CSVEntry> dbscanClustersInThisSecond) {
        CSVEntry dbscanHigherRandIndexEntry = null;
        double higherRandIndexValue = 0;
        double randIndexValue = 0;
        
        if (dbscanClustersInThisSecond != null) {
            for (CSVEntry dbscanEntry : dbscanClustersInThisSecond) {
                randIndexValue = randIndex(dg2cepEntry, dbscanEntry);
                if (higherRandIndexValue <= randIndexValue) {
                    higherRandIndexValue = randIndexValue;
                    dbscanHigherRandIndexEntry = dbscanEntry;
                }
            }
        }
        
        return dbscanHigherRandIndexEntry;
    }
    
    private double randIndex(CSVEntry dg2cepEntry, CSVEntry dbscanEntry) {
        if (dbscanEntry == null) { return 0; }
        
        Set<String> dg2cepSet = dg2cepEntry.getContent();
        Set<String> dbscanSet = dbscanEntry.getContent();
        
        Set<String> truePositive = new HashSet<String>(dg2cepSet);
        truePositive.retainAll(dbscanSet);
      
        Set<String> falsePositive = new HashSet<String>(dg2cepSet);
        falsePositive.removeAll(dbscanSet);

        Set<String> falseNegative = new HashSet<String>(dbscanSet);
        falseNegative.removeAll(dg2cepSet);
      
        double TP = truePositive.size();
        double FP = falsePositive.size();
        double FN = falseNegative.size();
          
        return TP / (TP + FP + FN);
    }
    
    private void calculateAllRandIndex() throws IOException {
        BufferedReader dg2cepBuffer = new BufferedReader(new FileReader(dg2cepFile));
        BufferedReader dbscanBuffer = new BufferedReader(new FileReader(dbscanFile));
        
        dg2cepSecBySecEntries = dstreamSecBySecParser(dg2cepBuffer);
        dbscanSecBySecEntries = dbscanSecBySecParser(dbscanBuffer);
        dg2cepBuffer.close();
        dbscanBuffer.close();
        
        int countTotalRandIndex = 0;
        int countTotalDetectedRandIndex = 0;
        int countTotalAdjRandIndex = 0;
        
        double totalRandIndex = 0;
        double totalDetectedRandIndex = 0;
        double totalNewRandIndex = 0;
        double totalAdjRandIndex = 0;
        
        long dbscanMinimumTimestamp = dbscanSecBySecEntries.firstKey();
        long dbscanMaximumTimestamp = dbscanSecBySecEntries.lastKey();
        long entryTimestamp = dbscanMinimumTimestamp;
        int previousNumberDBSCANClusters = 0;
        
        List<CSVEntry> dbscanClusters = new ArrayList<>();
        List<CSVEntry> dg2cepEntries  = null;
        List<CSVEntry> dbscanEntries  = null;
        
        ClusteringMeasurementTool adjustedRandIndex = ClusteringMeasurementTool.getInstance();
        randIndexFW.println("second,adjrandindex,randindex,randidexfound,dg2cepfound,dbscanfound");
        System.out.println("second,adjrandindex,randindex,randidexfound,dg2cepfound,dbscanfound");
        // Iterating
        while (entryTimestamp <= dbscanMaximumTimestamp) {
            dg2cepEntries = dg2cepSecBySecEntries.get(entryTimestamp);
            dbscanEntries = dbscanSecBySecEntries.get(entryTimestamp);

            int found = 0;
            int dbscanFound = 0;
            double roundRand = 0;
            double roundRandAdj = 0;
            double roundDetectedRandIndex = 0;
            
            if (dg2cepEntries.size() > 0) {
                if (dbscanEntries == null) {
                    countTotalRandIndex += dg2cepEntries.size();
                    countTotalAdjRandIndex += dg2cepEntries.size();
                } else {
                    roundRandAdj = adjustedRandIndex.adjustedRandIndex(dg2cepEntries, dbscanEntries);
                    totalAdjRandIndex+= roundRandAdj;
                    
                    countTotalAdjRandIndex += 1;
                    dbscanFound = dbscanEntries.size();
                    
                    for (CSVEntry dg2cepEntry : dg2cepEntries) {
                        CSVEntry dbscanEntry = getHigherRandIndex(dg2cepEntry, dbscanEntries);
                        double rand = randIndex(dg2cepEntry, dbscanEntry);
                        double newRandIndex = adjustedRandIndex.randIndex(dg2cepEntry, dbscanEntry);
                        
                        if (rand > randLowThreshold) {
                            totalDetectedRandIndex += rand;
                            countTotalDetectedRandIndex++;
                            found++;
                            roundDetectedRandIndex+= newRandIndex;
                        }
                        
                        roundRand+= newRandIndex;
                        totalNewRandIndex += newRandIndex;
                        totalRandIndex += randIndex(dg2cepEntry, dbscanEntry);
                        countTotalRandIndex++;
//                        System.out.println(adjustedRandIndex.randIndex(dg2cepEntry, dbscanEntry));
//                        System.out.println(dg2cepEntry.getSize() + "," + dbscanEntry.getSize() + "," + randIndex(dg2cepEntry, dbscanEntry));
                    }
                    
                    if (found != 0) {
                        roundDetectedRandIndex = roundDetectedRandIndex / found;    
                    } else {
                        roundDetectedRandIndex = 0;
                    }

                    roundRand = Math.min(1, roundRand / dbscanEntries.size());
                    if (found < dbscanEntries.size()) {
                        countTotalRandIndex += dbscanEntries.size() - found;
                    }
                }              
            } else {
                if (dbscanEntries == null) {
                    totalDetectedRandIndex += 1;
                    countTotalDetectedRandIndex += 1;
                    
                    totalRandIndex += 1;
                    countTotalRandIndex += 1;
                    
                    totalAdjRandIndex += 1;
                    totalNewRandIndex += 1;
                    countTotalAdjRandIndex += 1;
                    
                    roundRand = 1;
                } else {
                    dbscanFound = dbscanEntries.size();

                    countTotalRandIndex += dbscanEntries.size();
                    countTotalAdjRandIndex += dbscanEntries.size();
                    roundRand = 0;
                }
            }

            DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
            Calendar cal = Calendar.getInstance();
            int seconds = (int) (entryTimestamp  / 1000) % 60 ;
            int minutes = (int) ((entryTimestamp / (1000*60)) % 60);
            int hours   = (int) ((entryTimestamp / (1000*60*60)) % 24);
            hours+= 12;

            cal.set(Calendar.DAY_OF_MONTH, 18);
            cal.set(Calendar.MONTH, Calendar.JUNE);
            cal.set(Calendar.HOUR, hours);
            cal.set(Calendar.MINUTE, minutes);
            cal.set(Calendar.SECOND, seconds);

//          randIndexFW.println("second,adjrandindex,randindex,randidexfound,dg2cepfound,dbscanfound");
            System.out.println(dateFormat.format(cal.getTime()) + "," + roundRandAdj + "," + roundRand + ","
                               + roundDetectedRandIndex + "," + found + "," + dbscanFound);
            randIndexFW.println(dateFormat.format(cal.getTime()) + "," + roundRandAdj + "," + roundRand + ","
                                + roundDetectedRandIndex + "," + found + "," + dbscanFound);

            // Incrementing timestamp
            entryTimestamp += 1000;
            
            // DBSCAN Clusters
            if (dbscanEntries != null) {
                if (dbscanEntries.size() != previousNumberDBSCANClusters) {
                    for (CSVEntry dbscanEntry : dbscanEntries) {
                        updateClusters(dbscanEntry, dbscanClusters);
                    }
                    previousNumberDBSCANClusters = dbscanEntries.size();
                }
            }
            
        }
        
        // Number of Clusters detected.
        List<CSVEntry> finalClusters = new ArrayList<>();
        
        for (CSVEntry dbscanEntry : dbscanClusters) {
            boolean has = false;
            for (CSVEntry finalEntry : finalClusters) {
                if (randIndex(dbscanEntry, finalEntry) > 0) {
                    has = true;
                }
            }
            
            if (!has) {
                finalClusters.add(dbscanEntry);
            }
        }
        
        int numberdetected = 0;
        for (CSVEntry dbscanEntry : finalClusters) {
            boolean detected = false; 
            entryTimestamp = dbscanMinimumTimestamp;
            System.out.println(dbscanEntry);
            // Iterating
            while (entryTimestamp < dbscanMaximumTimestamp) {
                dg2cepEntries = dg2cepSecBySecEntries.get(entryTimestamp);
                if (dg2cepEntries.size() > 0) {
                    for (CSVEntry dg2cepEntry : dg2cepEntries) {
                        double rand = randIndex(dg2cepEntry, dbscanEntry);
                        if (rand > randLowThreshold) {
                            detected = true;
                            break;
                        }
                    }
                }
                if (detected) break;
                entryTimestamp += 1000;
            }
            
            if (detected) {
                numberdetected++;
            }
        }
        
        System.out.println("Number Detected: " + numberdetected + "/" + finalClusters.size());
        System.out.println("Rand Index Detected é: " + (totalDetectedRandIndex/countTotalDetectedRandIndex));
        System.out.println("Rand Index Total é: " + (totalRandIndex/countTotalRandIndex));
        System.out.println("New Rand Index Total é: " + (totalNewRandIndex/countTotalRandIndex));
        System.out.println("Adj Rand Index Total é: " + (totalAdjRandIndex/countTotalAdjRandIndex));
        randIndexFW.flush();
        randIndexFW.close();
    }
    
	/**
	 * Main Routine.
	 * 
	 * @param args a CSV filename.
	 * @throws IOException If log files cannot be created.
	 * @throws IllegalAccessException if args is null. 
	 */
	public static void main(String[] args) throws IOException {
	    if (args.length != 3) {
	        System.out.println("USAGE: java -jar CSVRandIndex dg2cepFile.csv dbscanFile.csv outputFile.csv");
	    }
	    
	    CSVDstreamRandIndex csvtool = new CSVDstreamRandIndex(args[0], args[1], args[2]);
	    csvtool.calculateAllRandIndex();
	}
}

//
//private void measureAddEntry(CSVEntry dg2cepEntry) {
//  long padded = dg2cepEntry.getTimestamp() / 1000;
//  long before = (padded) * 1000;
//  
//  CSVEntry dbscanBeforeEntry = null;
//  do {
//      dbscanBeforeEntry = getHigherRandIndex(dg2cepEntry, dbscanSecBySecEntries.get(before));
//      before -= 1000;
//  } while (randIndex(dg2cepEntry, dbscanBeforeEntry) > 0.6);
//  
//  int diff = (int) (dg2cepEntry.getTimestamp() - dbscanBeforeEntry.getTimestamp());
//  int ins = diff / 1000;
//  discoverFW.println(dg2cepEntry.getSize() + "," + diff);
//  System.out.println(dg2cepEntry.getSize() + "," + diff);
//}
//
//private void measureDisperseEntry(CSVEntry dg2cepEntry) {
//  long padded = dg2cepEntry.getTimestamp() / 1000;
//  long before = padded * 1000;
//  
//  CSVEntry dbscanBeforeEntry = null;
//  do {
//      dbscanBeforeEntry = getHigherRandIndex(dg2cepEntry, dbscanSecBySecEntries.get(before));
//      before -= 1000;
//  } while (randIndex(dg2cepEntry, dbscanBeforeEntry) < 0.1  && before > 0);
//  
//  if (dbscanBeforeEntry != null) {
//      int diff = (int) (dg2cepEntry.getTimestamp() - dbscanBeforeEntry.getTimestamp());
//      disperseFW.println(dg2cepEntry.getSize() + "," + diff);
//      System.out.println(dg2cepEntry.getSize() + "," + diff);
//  }
//}

